from urllib.request import urlopen, urlretrieve
from bs4 import BeautifulSoup
import os

downDirectory = 'download'
baseUrl = 'http://pythonscraping.com'


def getAbsoluteURL(baseUrl, source):
    if source.startswith('http://www.'):
        url = 'http://' + source[11:]
    elif source.startswith('http://'):
        url = source
    elif source.startswith('www.'):
        url = source[4:]
        url = 'http://' + source
    else:
        url = baseUrl + '/' + source

    if baseUrl not in url:
        return None

    return url


def getDownPath(baseUrl, absoluteUrl, downDirectory):
    path = absoluteUrl.replace('www.', '')
    path = path.replace(baseUrl, '')
    path = downDirectory + path
    directory = os.path.dirname(path)

    if not os.path.exists(directory):
        os.makedirs(directory)

    return path


html = urlopen('http://www.pythonscraping.com/')
bsObj = BeautifulSoup(html, 'html.parser')
downList = bsObj.findAll(src=True)
for download in downList:
    fileUrl = getAbsoluteURL(baseUrl, download['src'])

    if fileUrl is not None:
        print(fileUrl)
    urlretrieve(fileUrl, getDownPath(baseUrl, fileUrl, downDirectory))
